package com.itheima.webmagic.crawlerTask;

import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.ProxyProvider;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;

@Component
public class ProxyTest implements PageProcessor {

    //@Scheduled(fixedDelay = 1000 )
    public void process(){
        //代理
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("123.169.38.249", 808)));

        Spider.create(new ProxyTest())
                .addUrl("http://200019.ip138.com/")
                .setDownloader(httpClientDownloader)
                .run();
    }

    @Override
    public void process(Page page) {
        System.out.println(page.getHtml().css("p[align]").all().get(0).toString());

    }

    private Site site=Site.me().setTimeOut(1000*1000);
    @Override
    public Site getSite() {
        return site;
    }
}
